//package com.xiaohongshu.sns.slytherin.spider.work;
//
//import com.xiaohongshu.sns.slytherin.spider.client.HttpClientRequestContext;
//import com.xiaohongshu.sns.slytherin.spider.client.HttpUriRequestConverter;
//import com.xiaohongshu.sns.slytherin.spider.page.Page;
//import com.xiaohongshu.sns.slytherin.spider.proxy.Proxy;
//import com.xiaohongshu.sns.slytherin.spider.proxy.ProxyProvider;
//import com.xiaohongshu.sns.slytherin.spider.result.Request;
//import org.apache.commons.io.IOUtils;
//import org.apache.http.HttpResponse;
//import org.apache.http.client.methods.CloseableHttpResponse;
//import org.apache.http.impl.client.CloseableHttpClient;
//import org.apache.http.impl.client.HttpClients;
//import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
//import org.apache.http.util.EntityUtils;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//import us.codecraft.webmagic.Site;
//import us.codecraft.webmagic.Task;
//import us.codecraft.webmagic.downloader.HttpClientGenerator;
//import us.codecraft.webmagic.selector.PlainText;
//import us.codecraft.webmagic.utils.CharsetUtils;
//import us.codecraft.webmagic.utils.HttpClientUtils;
//
//import java.io.IOException;
//import java.nio.charset.Charset;
//import java.util.concurrent.TimeUnit;
//
//public class SlytherinDownloader2 extends AbstractDownloader {
//
//    private static final Logger logger = LoggerFactory.getLogger(SlytherinDownloader.class);
//
//    private final PoolingHttpClientConnectionManager connManager;
//    private final HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();
//    private final HttpClientGenerator httpClientGenerator;
//    private ProxyProvider proxyProvider;
//    private int maxPerRoute = 10; // 默认每个路由的最大连接数
//    private int maxTotalConnections = 50; // 默认总连接数
//
//    public SlytherinDownloader2() {
//        this.connManager = new PoolingHttpClientConnectionManager();
//        this.httpClientGenerator = new HttpClientGenerator(connManager).setPoolSize();
//    }
//
//    public void setPoolSize(int maxTotalConnections, int maxPerRoute) {
//        this.maxTotalConnections = maxTotalConnections;
//        this.maxPerRoute = maxPerRoute;
//        connManager.setMaxTotal(maxTotalConnections);
//        connManager.setDefaultMaxPerRoute(maxPerRoute);
//    }
//
//    public void setProxyProvider(ProxyProvider proxyProvider) {
//        this.proxyProvider = proxyProvider;
//    }
//
//    @Override
//    public Page download(Request request, Task task) {
//        if (task == null || task.getSite() == null) {
//            throw new NullPointerException("task or site can not be null");
//        }
//        Page page = Page.fail();
//        CloseableHttpResponse httpResponse = null;
//        CloseableHttpClient httpClient = getHttpClient(task.getSite());
//        Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(task) : null;
//        try {
//            HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy);
//            httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
//            page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
//            onSuccess(request);
//            logger.info("downloading page success {}", request.getUrl());
//        } catch (IOException e) {
//            logger.warn("download page {} error", request.getUrl(), e);
//            onError(request);
//        } finally {
//            if (httpResponse != null) {
//                try {
//                    EntityUtils.consumeQuietly(httpResponse.getEntity());
//                } catch (IOException e) {
//                    logger.error("Error consuming response entity", e);
//                }
//            }
//            if (proxyProvider != null && proxy != null) {
//                proxyProvider.returnProxy(proxy, page, task);
//            }
//        }
//        return page;
//    }
//
//    @Override
//    public void setThread(int threadNum) {
//        httpClientGenerator.setPoolSize(threadNum);
//    }
//
//    private CloseableHttpClient getHttpClient(Site site) {
//        return httpClientGenerator.getClient(site);
//    }
//
//    protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException {
//        byte[] bytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
//        String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue();
//        Page page = new Page();
//        page.setBytes(bytes);
//        if (!request.isBinaryContent()) {
//            if (charset == null) {
//                charset = getHtmlCharset(contentType, bytes);
//            }
//            page.setCharset(charset);
//            page.setRawText(new String(bytes, charset));
//        }
//        page.setUrl(new PlainText(request.getUrl()));
//        page.setRequest(request);
//        page.setStatusCode(httpResponse.getStatusLine().getStatusCode());
//        page.setDownloadSuccess(true);
//        return page;
//    }
//
//    private String getHtmlCharset(String contentType, byte[] contentBytes) throws IOException {
//        String charset = CharsetUtils.detectCharset(contentType, contentBytes);
//        if (charset == null) {
//            charset = Charset.defaultCharset().name();
//            logger.warn("Charset autodetect failed, use {} as charset. Please specify charset in Site.setCharset()", Charset.defaultCharset());
//        }
//        return charset;
//    }
//}